package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.main.ChengduLifeMain;
import com.spider.silence.main.Main;
import com.spider.silence.main.TianFuForumMain;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.TianFuForumModuleTask;
import com.spider.silence.task.TianFuForumTask;
import com.spider.silence.task.YiBinZeroJuLiTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by Administrator on 2016/12/28 0028.
 */
public class TianFuForumDownLoader extends AbstractCommonDownLoader implements Cloneable,Runnable{
    private Logger logger = Logger.getLogger(KaiDiDownLoader.class);

    private String url = "http://bbs.scol.com.cn/";

    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }
    public List<String> getUrl(){
        List<String> urlList = new ArrayList<String>();

        Document document = null;
        try {
            document = get(url);

            Element element=document.body();
            Elements urls = element.select(".header").select(".sub-nav2").select("a");
            Elements cityForumUrls = element.select(".header").select(".sub-nav3").select("a");
            for(int i =0;i<urls.size();i++){
                String url = urls.get(i).attr("href");
                urlList.add(url);

            }
            for(int j = 0;j<cityForumUrls.size();j++){
                String url = cityForumUrls.get(j).attr("href");
                urlList.add(url);
            }

            System.out.println(urlList);
        } catch (Exception e) {
            try {
                document = get(url, true);
                Element element=document.body();
                Elements urls = element.select(".header").select(".sub-nav2").select("a");
                Elements cityForumUrls = element.select(".header").select(".sub-nav3").select("a");
                for(int i =0;i<urls.size();i++){
                    String url = urls.get(i).attr("href");
                    urlList.add(url);

                }
                for(int j = 0;j<cityForumUrls.size();j++){
                    String url = cityForumUrls.get(j).attr("href");
                    urlList.add(url);
                }

                System.out.println(urlList);
            }catch (Exception e1){
                Logger logger = Logger.getLogger(TianFuForumMain.class);
                logger.error("获取论坛版块url出错");
                logger.error(e.getMessage(),e);
            }

            //e.printStackTrace();
        }

        return urlList;
    }

    public void run(){

    }

    /**
     * 爬取历史数据
     * @param urlList
     */
    public void generateAllTask(List<String> urlList){
        int a=0;
        for (int i = 0; i < urlList.size(); i++) {
            boolean flag = true;
            Document document = null;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while(flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("TianFuForum.WaitTime"))) {
                try {
//                    String proxy=list.get(new Random().nextInt(list.size()))[2]+":"+list.get(new Random().nextInt(list.size()))[3];
                    Page pageContent = getHtml(urlList.get(i));
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 200) {
                        flag=false;
                        document = pageContent.getHtml().getDocument();
                        int page = 0;
                        if (document.select("#fd_page_top").select(".pg").size() == 0) {
                            page = 1;
                        } else {
                            page = Integer.parseInt(document.select("#fd_page_top").select(".pg").select("label").select("span").get(0).attr("title").split(" ")[1]); //页码
                        }
                        for (int j = 1; j < page + 1; j++) {
                            try {
                                Document d = get(urlList.get(i).split("-")[0] + "-" + urlList.get(i).split("-")[1] + "-" + j + ".html");
                                if("http://bbs.scol.com.cn/forum-2550-1.html".equals(urlList.get(i))){  //天府旅游
                                    Elements element = d.select(".c.cl");
                                    if (element.size() > 0) {
                                        for (int k = 0; k < element.size(); k++) {
                                            String url = element.get(k).select("a").attr("href");
                                            Document doc=get(url);
                                            if (doc.title() == null || "".equals(doc.title())) {
                                                String design = getDsign(url);
                                                url = url + "?_dsign="+design;
                                            }
                                            TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(url, "UTF-8"));
                                            CommonTaskManager.add("TianFuForum", task);
                                        }
                                    }
                                }else {
                                    Elements element = d.select(".s.xst");
                                    if (element.size() > 0) {
                                        for (int k = 0; k < element.size(); k++) {
                                            String url = element.get(k).attr("href");
                                            Document doc=get(url);
                                            if (doc.title() == null || "".equals(doc.title())) {
                                                String design = getDsign(url);
                                                url = url + "?_dsign="+design;
                                            }
                                            TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(url, "UTF-8"));
                                            CommonTaskManager.add("TianFuForum", task);
                                        }
                                    }
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                                logger.error(e.getMessage(),e);
                                continue;
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    Logger logger = Logger.getLogger(ChengduLifeMain.class);
                    logger.error(e.getMessage(),e);
                }finally {
                    nowTime = System.currentTimeMillis();
                }
            }


        }
    }

    /**
     *生成Task 并添加到TaksManager
     */
    public void generateTask(List<String> urlList){
        for(int i=0;i<urlList.size();i++){
            /*ExecutorService pool = Executors.newFixedThreadPool(10);

            try {
                if("http://bbs.scol.com.cn/forum-2550-1.html".equals(urlList.get(i))){
                    List<Elements> elementsList = new ArrayList<Elements>();
                    Document document = get(urlList.get(i));
                    Elements imgList = document.getElementById("waterfall").select("li");
                    String forum_url = urlList.get(i);

                    elementsList.add(imgList);

                    *//*for(int j = 0;j<imgList.size();j++){
                        String forumUrl = imgList.get(j).select(".c").select("a").get(0).attr("href");
                        String design = getDsign(forumUrl);
                        forumUrl = forumUrl + "?_dsign="+design;
                        TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(forumUrl, "UTF-8"));
                        CommonTaskManager.add("TianFuForum", task);
                        System.out.println(forumUrl);
                    }*//*

                    *//*String total = document.select("#pgt").select("#fd_page_bottom").select(".pg").select("label").text();
                    Pattern pattern = Pattern.compile("[^0-9]");
                    Matcher matcher = pattern.matcher(total);
                    String all = matcher.replaceAll("");
                    System.out.println("page:" + all);
                    int total_page = Integer.valueOf(all);
                    for(int j = 2;j<=total_page;j++){
                        try {
                            String new_url = forum_url.split("-")[0] + "-" + forum_url.split("-")[1] + "-" + j + ".html";
                            Document document1 = get(new_url);
                            Elements imgList1 = document1.getElementById("waterfall").select("li");
                            elementsList.add(imgList1);
                        }catch (Exception e){
                            continue;
                        }
                    }*//*
                    ArrayList<ArrayList<Integer>> taskSplit = CommonUtils.splitPages(1, elementsList.size(), 10);
                    for(int j = 0;j<taskSplit.size();j++){
                        ImgModuleCrawler crawler = new ImgModuleCrawler(elementsList, taskSplit.get(i));
                        pool.execute(crawler);//提交任务
                    }

                }else {
                    //List<Elements> elementsList = new ArrayList<Elements>();
                    Document document = get(urlList.get(i));
                    Elements threadList = document.select("tbody[id^='normalthread_'] a.xst");
                    String forum_url = urlList.get(i);
                    ArrayList<ArrayList<Integer>> taskSplit = CommonUtils.splitPages(1, threadList.size(), 5);

                    //elementsList.add(threadList);
                   for(int j = 0;j<taskSplit.size();j++){
                       ModuleCrawler crawler = new ModuleCrawler(threadList, taskSplit.get(i));
                       pool.execute(crawler);//提交任务
                   }


                    *//*for (int j = 0; j < threadList.size(); j++) {
                        String forumUrl = threadList.get(j).attr("href");
                        String design = getDsign(forumUrl);
                        forumUrl = forumUrl + "?_dsign="+design;
                        TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(forumUrl, "UTF-8"));
                        CommonTaskManager.add("TianFuForum", task);
                        System.out.println(forumUrl);
                    }*//*


                    String total = document.select("#fd_page_bottom").select(".pg").select(".last").text();
                    Pattern pattern = Pattern.compile("[^0-9]");
                    Matcher matcher = pattern.matcher(total);
                    String all = matcher.replaceAll("");
                    System.out.println("page:" + all);
                    int total_page = Integer.valueOf(all);

                    //int subUrlThread = Integer.valueOf(PropertiesUtil.getVal("TianFuForum.SubUrlThreadCount"));

                    ExecutorService pool2 = Executors.newFixedThreadPool(20);
                    ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, total_page, 20);
                    if(total_page>2) {
                        for (int k = 0;k<splitPages.size();k++) {
                            try {
                               // String new_url = forum_url.split("-")[0] + "-" + forum_url.split("-")[1] + "-" + k + ".html";
                           *//* Document document1 = get(new_url);
                            Elements threadList1 = document1.select("tbody[id^='normalthread_'] a.xst");
                            elementsList.add(threadList1);*//*
                            *//*for (int j = 0; j < threadList1.size(); j++) {
                                String forumUrl1 = threadList1.get(j).attr("href");
                                String design = getDsign(forumUrl1);
                                forumUrl1 = forumUrl1 + "?_dsign="+design;
                                TianFuForumTask task1 = new TianFuForumTask(URLDecoder.decode(forumUrl1, "UTF-8"));
                                CommonTaskManager.add("TianFuForum", task1);
                                System.out.println(forumUrl1);
                            }*//*
                                SubForumUrl sub = new SubForumUrl(forum_url,splitPages.get(k));
                                //FutureTask<List<Elements>> futureTaskList = new FutureTask<List<Elements>>(callable);
                                //List<Elements> list = futureTaskList.get();
                               *//* for(int n = 0;n<list.size();n++){
                                    elementsList.add(list.get(n));
                                }*//*
                                pool2.execute(sub);

                                *//*for(FutureTask<List<Elements>> futuretask:futureTasks){
                                    List<Elements> list = futuretask.get();
                                    for(int n = 0;n<list.size();n++){
                                        elementsList.add(list.get(n));
                                    }
                                }*//*

                            } catch (Exception e) {
                                Logger logger = Logger.getLogger(Main.class);
                                logger.error("解析url出错", e);
                                continue;
                            }
                        }
                    }



                    *//*ArrayList<ArrayList<Integer>> taskSplit = CommonUtils.splitPages(1, elementsList.size(), 10);
                    for(int j = 0;j<taskSplit.size();j++){
                        ModuleCrawler crawler = new ModuleCrawler(elementsList, taskSplit.get(i));
                        pool.execute(crawler);//提交任务
                    }*//*


                }

            } catch (Exception e) {
                Logger logger = Logger.getLogger(TianFuForumMain.class);
                logger.error("获取板块中的帖子url出错",e);
                //e.printStackTrace();
                continue;
            }*/

            try {
                TianFuForumModuleTask task = new TianFuForumModuleTask(urlList.get(i));
                CommonTaskManager.add("TianFuForumModule", task);
            }catch (Exception e){
                logger.error(e.getMessage(),e);
            }
        }

    }



    class SubForumUrl extends AbstractCommonDownLoader implements Runnable{
        private Logger logger = Logger.getLogger(SubForumUrl.class);
        private String forum_url;
        private ArrayList<Integer> pages;


        public SubForumUrl(String forum_url,ArrayList<Integer> pages) {
            this.forum_url = forum_url;
            this.pages = pages;
        }

        @Override
        public void  run() {
            for (int k = 0; k < pages.size(); k++) {
                Elements threadList1 = null;
                try {
                    String new_url = forum_url.split("-")[0] + "-" + forum_url.split("-")[1] + "-" + pages.get(k) + ".html";
                    //String new_url = forum_url.split("-")[0] + "-" + forum_url.split("-")[1] + "-" + k + ".html";
                    Document document1 = get(new_url);
                    threadList1 = document1.select("tbody[id^='normalthread_'] a.xst");

                    for(int j = 0;j<threadList1.size();j++) {
                        String forumUrl = threadList1.get(j).attr("href");
                        String design = getDsign(forumUrl);
                        forumUrl = forumUrl + "?_dsign=" + design;
                        TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(forumUrl, "UTF-8"));
                        CommonTaskManager.add("TianFuForum", task);
                        System.out.println(forumUrl);
                    }


                } catch (Exception e) {
                    logger.error("解析url出错", e);
                    continue;
                }

            }

        }
    }

    class ModuleCrawler extends AbstractCommonDownLoader implements Runnable {
        private Logger logger = Logger.getLogger(ModuleCrawler.class);
        private Elements thread;
        private ArrayList<Integer> taskIndex;

        public ModuleCrawler(Elements thread, ArrayList<Integer> taskIndex) {
            this.thread = thread;
            this.taskIndex = taskIndex;
        }

        @Override
        public void run() {
            for (int k = 0; k < taskIndex.size(); k++) {
                //int index = taskIndex.get(k) - 1;
                //String url = elements.get(index).attr("href");
                try {
                    //Document document = get(url);
                   //Elements forumElements = document.select("tbody[id^='normalthread_'] a.xst");
                    for(int j = 0;j<thread.size();j++) {
                        String forumUrl = thread.get(j).attr("href");
                        String design = getDsign(forumUrl);
                        forumUrl = forumUrl + "?_dsign=" + design;
                        TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(forumUrl, "UTF-8"));
                        CommonTaskManager.add("TianFuForum", task);
                        System.out.println(forumUrl);
                    }

                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    e.printStackTrace();
                }
            }
        }
    }

    class ImgModuleCrawler extends AbstractCommonDownLoader implements Runnable{
        private Logger logger = Logger.getLogger(ImgModuleCrawler.class);
        private List<Elements> imgList;
        private ArrayList<Integer> taskIndex;

        public ImgModuleCrawler(List<Elements> imgList, ArrayList<Integer> taskIndex) {
            this.imgList = imgList;
            this.taskIndex = taskIndex;
        }

        @Override
        public void run() {
            for (int k = 0; k < taskIndex.size(); k++) {
                try {
                    for (int i = 0; i < imgList.size(); i++){
                        Elements imgs = imgList.get(i);
                        for(int j = 0;j<imgs.size();j++){
                            String forumUrl = imgs.get(j).select(".c").select("a").get(0).attr("href");
                            String design = getDsign(forumUrl);
                            forumUrl = forumUrl + "?_dsign="+design;
                            TianFuForumTask task = new TianFuForumTask(URLDecoder.decode(forumUrl, "UTF-8"));
                            CommonTaskManager.add("TianFuForum", task);
                            System.out.println(forumUrl);
                        }
                    }
                }catch (Exception e){
                    logger.error(e.getMessage(), e);
                    e.printStackTrace();
                }
            }
        }
    }


    public JSONObject crawlItem(String content_url){
        files=new ConcurrentHashMap<>();
        JSONObject item=new JSONObject();
        JSONObject data = new JSONObject();
        int page=1;
        //content_url = "http://bbs.scol.com.cn/thread-15156247-1-1.html?_dsign=a00b49ef";
       // long starTime = System.currentTimeMillis();
        boolean flag = true;

        long future= System.currentTimeMillis()+Long.valueOf(PropertiesUtil.getVal("TianFuForum.WaitTime"));
        long remianing=Long.valueOf(PropertiesUtil.getVal("TianFuForum.WaitTime"));
        //while(flag && remianing>=0) {

            try {
                //remianing=future-System.currentTimeMillis();
                Page htmlPage = new Page();
                htmlPage = getHtml(content_url);

                Document document = null;

                if (htmlPage.getRawText() == null || htmlPage.getStatusCode() != 200) {
                    document = get(content_url);
                } else {
                    document = htmlPage.getHtml().getDocument();
                    String title = document.title();
                    if (document.title() == null || "".equals(document.title())) {
                        document = get(content_url);
                    }

                }

            /*try {
                document = get(url, true);
            }catch (Exception e){
                url = url+"?_dsign="+getRealUrl(document.toString());
                document = get(url);
                //title = document.title();
            }*/

                String title = document.title();
                Element pageInfo = null;
                Elements countInfo = new Elements();
                String url1 = "";
                try {
                    pageInfo = document.getElementById("postlist");
                    countInfo = pageInfo.select(".floor").get(0).select(".authi").select(".cp_pls").select("span");
                } catch (Exception e) {
                    Logger logger = Logger.getLogger(TianFuForumMain.class);
                    //logger.error("解析帖子url出错");
                    document = get(content_url);
                    pageInfo = document.getElementById("postlist");
                    title = document.title();
                    countInfo = pageInfo.select(".floor").get(0).select(".authi").select(".cp_pls").select("span");
                }
                document.select(".aimg_tip").remove();
                int read_count = 0;
                int reply_count = 0;
                if (countInfo != null) {
                    try {
                        String readCount = countInfo.get(2).text();
                        String replyCount = countInfo.get(5).text();
                        if ("查看:".equals(readCount)) {
                            readCount = countInfo.get(3).text();
                            replyCount = countInfo.get(6).text();
                        }
                        if (readCount != null && !"".equals(readCount)) {
                            read_count = Integer.parseInt(readCount);
                        }
                        if (replyCount != null && !"".equals(replyCount)) {
                            reply_count = Integer.parseInt(replyCount);
                        }
                    }catch (Exception e){

                    }

                }
                page = (int) Math.ceil(((double) reply_count + 1) / 10d);//页数
                String id = content_url.split("//")[1].split("-")[1];//帖子id
                data.put("id", id);
                data.put("title", title);
                data.put("url", content_url);
                data.put("reply_count", reply_count);
                data.put("read_count", read_count);

                Elements contentInfo = pageInfo.select(".floor");

                Element userInfo = contentInfo.get(0);
                String content_id = userInfo.select("table").get(0).attr("id").substring(3);//言论id
                String publish_user_id = userInfo.select(".authi").select(".avatar").select("a").get(0).attr("href").split("//")[1].split("-")[2];
                String publish_user = userInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                String publish_user_photo = userInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                if (publish_user_id != null && !"".equals(publish_user_id)) {
                    publish_user_id = publish_user_id.split("\\.")[0];
                }


                data.put("content_id", content_id);
                data.put("publish_user_id", publish_user_id);
                data.put("publish_user", publish_user);
                data.put("publish_user_photo", publish_user_photo);

                String publish_time = "";

                if (userInfo.select(".authi").select(".cp_pls").select("em") != null) {
                    publish_time = userInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1] + " " + userInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2] + ":00";
                }
                data.put("publish_time", publish_time);
                Date now = new Date();
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                data.put("spider_time", sdf.format(now));

                String content = userInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                JSONArray img_urls=new JSONArray();
                Elements imgElements=userInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                    Element imgElement=imgElements.get(imgIndex);
                    String img_url=imgElement.absUrl("zoomfile");
                    img_urls.add(img_url);
                }
                data.put("img_urls",img_urls);
                int like_count = 0;
                int dislike_count = 0;
                if (userInfo.getElementById("recommendv_add") != null) {
                    like_count = Integer.parseInt(userInfo.getElementById("recommendv_add").text());
                    data.put("like_count", like_count);
                }
                if (userInfo.getElementById("recommendv_subtract") != null) {
                    dislike_count = Integer.parseInt(userInfo.getElementById("recommendv_subtract").text());
                    data.put("dislike_count", dislike_count);
                }

                data.put("content", FileUtils.filterImg(content));
                long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                String filePrefix="TianFuForum";//文件前缀
                String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
                String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                String speechesId="TianFuForum_speeches_"+publishDate+"_"+fileName;
                File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                files.put(speechesId+"_pageNo_1",file);
                //帖子首页的文件

                JSONArray reply_nodes = new JSONArray();


                //将本页面的回复添加上去
                if (contentInfo.size() > 1) {
                    for (int i = 1; i < contentInfo.size(); i++) {
                        JSONObject reply = new JSONObject();
                        Element replyInfo = contentInfo.get(i);
                        String reply_id = replyInfo.select("table").get(0).attr("id").substring(3);
                        String reply_user_id = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href").split("//")[1].split("-")[2].split("\\.")[0];
                        String reply_user_photo = replyInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                        String reply_publish_user = replyInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                        String reply_publish_time = "";
                        if (replyInfo.select(".authi").select(".cp_pls").select("em") != null) {
                            reply_publish_time = replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1] + " " + replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2] + ":00";
                        }
                        String homeUrl = "http://bbs.scol.com.cn/forum.php?mod=viewthread&tid=" + id + "&page=1#pid" + reply_id;

                        String reply_content = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                        JSONArray reply_img_urls=new JSONArray();
                        Elements replyImgElements=replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                        for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                            Element imgElement=replyImgElements.get(imgIndex);
                            String img_url=imgElement.absUrl("zoomfile");
                            reply_img_urls.add(img_url);
                        }
                        Date reply_now = new Date();
                        reply.put("spider_time", sdf.format(reply_now));
                        reply.put("id", reply_id);
                        reply.put("publish_user_id", reply_user_id);
                        reply.put("publish_user_photo", reply_user_photo);
                        reply.put("publish_user", reply_publish_user);
                        reply.put("publish_time", reply_publish_time);
                        reply.put("url", homeUrl);
                        reply.put("content", FileUtils.filterImg(reply_content));
                        reply.put("img_urls",reply_img_urls);

                        try {
                            if (replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote") != null && replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").size() > 0) {
                                if(replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a")!=null&&replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a").size()>0) {
                                    String reply_user_url = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a").get(0).attr("href");
                                    String parent_id = getUrlParams(reply_user_url).get("pid").toString();
                                    reply.put("parent_id", parent_id);
                                    reply.put("ancestor_id", id);
                                }else{
                                    reply.put("parent_id", id);
                                    reply.put("ancestor_id", id);
                                }
                            } else {
                                reply.put("parent_id", id);
                                reply.put("ancestor_id", id);
                            }
                        }catch (Exception e){
                            Logger logger = Logger.getLogger(Main.class);
                            logger.error("取父节点id出错",e);
                            reply.put("parent_id", id);
                            reply.put("ancestor_id", id);
                        }

                        reply_nodes.add(reply);
                    }
                }
                JSONArray replys = new JSONArray();
                List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                if (page >= 2) {
                    int threadNum = Integer.valueOf(PropertiesUtil.getVal("TianFuForum.ReplyThreadCount"));
                    ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                    try {
                        List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                        ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                        for (int i = 0; i < splitPages.size(); i++) {
                            TianFuForumDownLoader downLoader = this.cloneDownLoader();
                            ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), content_url, id,speechesId);
                            FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                            futureTasks.add(futureTask);
                            executorService.submit(futureTask);
                        }

                        int count = 0;
                        for (FutureTask<JSONArray> futureTask : futureTasks) {
                            JSONArray replyArrays = futureTask.get();
                            for (int i = 0; i < replyArrays.size(); i++) {
                                JSONObject object = replyArrays.getJSONObject(i);
                                replys.add(object);
                            }
                        }
                        if (replys.size() > 0 && replys != null) {
                            for (int i = 0; i < replys.size(); i++) {
                                JSONObject reply_object = replys.getJSONObject(i);
                                reply_nodes.add(reply_object);
                            }
                        }
                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                            for (int i = 0; i < reply_nodes.size(); i++) {
                                JSONObject reply_object = reply_nodes.getJSONObject(i);
                                Map<String, Object> map = reply_object;
                                list.add(map);
                            }
                        }
                        Collections.sort(list, new Comparator<Map<String, Object>>() {

                            public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                String s1 = o1.get("publish_time").toString();
                                String s2 = o2.get("publish_time").toString();
                                int flag = s1.compareTo(s2);
                                return flag;
                            }
                        });
                    }catch (Exception e){
                        logger.error(e.getMessage(),e);
                    }finally {
                        executorService.shutdown();
                    }
                } else {
                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                        for (int i = 0; i < reply_nodes.size(); i++) {
                            JSONObject reply_object = reply_nodes.getJSONObject(i);
                            Map<String, Object> map = reply_object;
                            list.add(map);
                        }
                    }
                }

                data.put("reply_nodes", list);
                flag = false;
            } catch (Exception e) {
                Logger logger = Logger.getLogger(TianFuForumMain.class);
                logger.error("解析帖子内容出错",e);
                //e.printStackTrace();
            }

            item.put("data", data);

        //}
        return item;
    }


    public static void main(String[] args){
        TianFuForumDownLoader downloader = new TianFuForumDownLoader();
        /*List<String> urlList = downloader.getUrl();
        downloader.generateTask(urlList);*/
        downloader.crawlItem("http://bbs.scol.com.cn/thread-15183260-1-1.html?_dsign=1d172f4a");
    }

    public TianFuForumDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        TianFuForumDownLoader loader =new TianFuForumDownLoader();
        loader.cookies= (HashSet<Cookie>) this.cookies.clone();
        loader.headers= (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public String getRealUrl(String script){
        //String script = "<script type=\"text/javascript\">_B7gtF = window;_cuUaN = 'href';function T7(){'return T7';return '.'}BF=function(){'BF';var _B=function(){return 'd'}; return _B();};_BDeka = 'assign';function getName(){var caller=getName.caller;if(caller.name){return caller.name} var str=caller.toString().replace(/[\\s]*/g,\"\");var name=str.match(/^function([^\\(]+?)\\(/);if(name && name[1]){return name[1];} else {return '';}}function A49(){'A49';function _A(){return '5a'}; return _A();}zw=function(){'zw';var _z=function(){return 'a'}; return _z();};Ti9I=function(){'return Ti9I';return 'htm';};i8=function(){'i8';var _i=function(){return '5'}; return _i();};Zs=function(){'Zs';var _Z=function(){return 'h'}; return _Z();};function cU(){'cU';function _c(){return '/'}; return _c();}wbS=function(wbS_){'return wbS';return wbS_;};Ix2S='6-1';function UL(){'UL';function _U(){return '9'}; return _U();}_tDmDC = 'replace';Ta=function(){'return Ta';return 'g';};V89P=function(){'V89P';var _V=function(){return '587'}; return _V();};a3='d';qO=function(){'return qO';return 'l';};c95A=function(){'c95A';var _c=function(){return '151'}; return _c();};function n4(){'return n4';return '-'}_UuqKg = location;location[_cuUaN]=cU()+(function(){'return tf';return (function(){return 't';})();})()+Zs()+'re'+zw()+BF()+n4()+c95A()+i8()+V89P()+Ix2S+'-1'+T7()+Ti9I()+qO()+(function(z1u_){return (function(z1u_){return z1u_;})(z1u_);})('?_')+a3+(function(ASS_){'return ASS';return ASS_})('si')+Ta()+(function(){'return Ltly';return (function(){return 'n=e';})();})()+A49()+(function(fKM_){return (function(fKM_){return fKM_;})(fKM_);})('cc')+UL()+wbS('7d');_B7gtF[_cuUaN]=cU()+(function(){'return tf';return (function(){return 't';})();})()+Zs()+'re'+zw()+BF()+n4()+c95A();</script>";
        String new_script = script;

        String[] strs = {"window","location","'assign'","'href'","'replace'"};

        //findall("[_A-Za-z0-9 =]+%s;"+%st,js)
        List<String> equal = new ArrayList<String>();


        for(int i = 0;i<strs.length;i++) {
            String regex = String.format("[_A-Za-z0-9 =]+%s;",strs[i]);
            //System.out.println(new_script.indexOf(regex));
            Pattern p = Pattern.compile(regex);
            Matcher m = p.matcher(new_script);
            // ArrayList<String> strs1 = new ArrayList<String>();
            while (m.find()) {
                equal.add(m.group(0));
                //System.out.println(m.group(0));
            }

        }

        if(equal.size()>0&&equal!=null) {
            for (String s : equal) {
                String varname = s.split("=")[0].trim();
                new_script = new_script.replaceAll(s,"");
                String var = s.split("=")[1];
                var = var.substring(0,var.length()-1);
                new_script = new_script.replaceAll(varname,var);
            }
        }

        List<String> equal1 = new ArrayList<String>();

        for(int i = 0;i<strs.length;i++){
            String regex1 = String.format("\\[\\s*%s\\]",strs[i]);
            String new_replace = String.format(".%s",strs[i]);
            Pattern p = Pattern.compile(regex1);
            Matcher m = p.matcher(new_script);
            while (m.find()) {
                equal1.add(m.group(0));

            }

            new_replace = new_replace.replaceAll("\'","");
            for (String s : equal1) {
                new_script = new_script.replace(s,new_replace);
                //System.out.println(s);
            }
            //new_script = new_script.replaceAll(regex1,new_replace);
        }

        new_script = new_script.replaceAll("location\\.href=","var url=");
        new_script = new_script.replaceAll("window\\.href=","var url2=");
        new_script = new_script.replaceAll("location\\.assign","var url=");
        new_script = new_script.replaceAll("location\\.replace","var url=");
        new_script = new_script.replaceAll("location=","var url=");

        //String regex2 = "return getName()";
        new_script = new_script.replaceAll("return getName\\(\\)","return ''");

        //System.out.println(new_script);

        ScriptEngineManager manager = new ScriptEngineManager();
        ScriptEngine engine = manager.getEngineByName("javascript");
        String url = "";
        try{
           // System.out.println("var location={};"+new_script);
            url= (String) engine.eval(new_script+";function rtn(){return url;}rtn();");
            System.out.println(url);
            // engine.eval("alert(\"js alert\");");    // 不能调用浏览器中定义的js函数 // 错误，会抛出alert引用不存在的异常
        }catch(ScriptException e) {

            e.printStackTrace();
        }
        /*Object obj = engine.get("url");
        Invocable inv = (Invocable) engine;

        System.out.println(obj);*/
        String url2 = "";
        if(url!=null){
            url2 = url.split("\\?")[1].split("=")[1];
        }else{
            url2 = "";
        }
        /*if(url2.length()==8){
            return url2;
        }else{
            return "";
        }*/
        return url2;
        // System.out.println(new_script);
    }

    public String getDsign(String url) throws Exception {
        String no="";
        while (true){

            String aa = "";
            try {
                aa = getString(url);
            }catch(Exception e){
                aa = getString(url);
            }

            if(aa!=null && !"".equals(aa)) {
                aa = aa.substring(31, aa.length() - 9);
                aa = "var cc='';\n" +
                        "getName.caller={};\n" +
                        "var window=function () {};\n" +
                        "var location=function () {};\n" +
                        "window.prototype.href='';\n" +
                        "location.prototype.href=window.prototype.href;\n" +
                        "location.replace=function(str){ if(str.length>0){cc=str}else{cc=window.href;}};\n" +
                        "location.assign=function(str){ if(str.length>0){cc=str}else{cc=window.href;}};" + aa + "function test(){return cc;};test();";
                ScriptEngineManager manager = new ScriptEngineManager();
                ScriptEngine engine = manager.getEngineByName("javascript");
                try {
                    String cc = (String) engine.eval(aa);
                    if (!cc.isEmpty() && cc.split("=")[1].length() == 8) {
                        no = cc.split("=")[1];
                        break;
                    } else {
                        continue;
                    }
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
        }
        return no;
    }




    public class ReplyCrawler implements Callable<JSONArray> {
        private TianFuForumDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String content_id;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();
            for(int k=0;k<pages.size();k++){
                try {
                    String getUrl = url.split("-")[0]+"-"+url.split("-")[1]+"-"+pages.get(k)+"-"+url.split("-")[3];
                    Document document = downLoader.get(getUrl);
                    File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                    files.put(speeches_id+"_pageNo_"+pages.get(k),file);

                    String id = url.split("//")[1].split("-")[1];
                    String title=document.title();
                    if(title ==null ||"".equals(title)){
                        String url1 = url+"?_dsign="+downLoader.getRealUrl(document.select("script").get(0).html());
                        document = downLoader.get(url1);
                        title = document.title();
                    }

                    document.select(".aimg_tip").remove();
                    Element pageInfo = null;
                    Elements replyInfos = new Elements();
                    try {
                        pageInfo = document.getElementById("postlist");
                        replyInfos = pageInfo.select(".floor");
                    }catch (Exception e){
                        Logger logger = Logger.getLogger(TianFuForumMain.class);
                        logger.error("访问帖子的回复内容出错");
                        document = get(url,true);
                        pageInfo = document.getElementById("postlist");
                        replyInfos =  pageInfo.select(".floor");
                    }

                    for(int i =0;i<replyInfos.size();i++){
                        try{
                            JSONObject reply = new JSONObject();
                            Element replyInfo = replyInfos.get(i);
                            String reply_id = replyInfo.select("table").get(0).attr("id").substring(3);
                            String reply_user_id = replyInfo.select(".authi").select(".avatar").select("a").get(0).attr("href").split("//")[1].split("-")[2].split("\\.")[0];
                            String reply_user_photo = replyInfo.select(".authi").select(".avatar").select("a").get(0).select("img").get(0).attr("src");
                            String reply_publish_user =  replyInfo.select(".authi").select(".floor-profile").select("a").get(0).text();
                            String reply_publish_time = "";
                            if(replyInfo.select(".authi").select(".cp_pls").select("em")!=null){
                                reply_publish_time = replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[1]+" "+replyInfo.select(".authi").select(".cp_pls").select("em").text().split(" ")[2]+":00";
                            }
                            String homeUrl = "http://bbs.scol.com.cn/forum.php?mod=viewthread&tid="+id+"&page="+pages.get(k)+"#pid"+reply_id;

                            String reply_content = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").text();
                            JSONArray reply_img_urls=new JSONArray();
                            Elements replyImgElements=replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select("img[zoomfile]");
                            for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                Element imgElement=replyImgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("zoomfile");
                                reply_img_urls.add(img_url);
                            }
                            Date reply_now = new Date();
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                            reply.put("spider_time",sdf.format(reply_now));
                            reply.put("id",reply_id);
                            reply.put("publish_user_id",reply_user_id);
                            reply.put("publish_user_photo",reply_user_photo);
                            reply.put("publish_user",reply_publish_user);
                            reply.put("publish_time",reply_publish_time);
                            reply.put("url",homeUrl);
                            reply.put("content",FileUtils.filterImg(reply_content));
                            reply.put("img_urls",reply_img_urls);
                            try {
                                if (replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote") != null && replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").size() > 0) {
                                    if(replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a")!=null&&replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a").size()>0) {
                                        String reply_user_url = replyInfo.select(".pcb").select(".t_fsz").select(".t_f").select(".quote").select("a").get(0).attr("href");
                                        String parent_id = getUrlParams(reply_user_url).get("pid").toString();
                                        reply.put("parent_id", parent_id);
                                        reply.put("ancestor_id", content_id);
                                    }else{
                                        reply.put("parent_id", content_id);
                                        reply.put("ancestor_id", content_id);
                                    }

                                } else {
                                    reply.put("parent_id", content_id);
                                    reply.put("ancestor_id", content_id);
                                }
                            }catch (Exception e){
                                Logger logger = Logger.getLogger(Main.class);
                                logger.error("取父节点id出错",e);
                                reply.put("parent_id", content_id);
                                reply.put("ancestor_id", content_id);
                            }
                            array.add(reply);
                        }catch (Exception e){
                            continue;//因为禁言无法获取的情况
                        }
                    }
                } catch (Exception e) {
                    Logger logger = Logger.getLogger(TianFuForumMain.class);
                    logger.error("解析帖子内容出错");
                    //e.printStackTrace();
                    continue;
                }
            }
            return array;
        }
        public ReplyCrawler(TianFuForumDownLoader downLoader, ArrayList<Integer> pages,String url,String content_id,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url=url;
            this.content_id = content_id;
            this.speeches_id=speeches_id;
        }
    }

}
